# -*- coding: utf-8 -*-

# Scrapy settings for blog project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'blog'

SPIDER_MODULES = ['blog.spiders']
NEWSPIDER_MODULE = 'blog.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'blog (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 0.3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36",

"cookie": "_ga=GA1.2.302134352.1566468665; _gid=GA1.2.1599621326.1566468665; UM_distinctid=16cbc6055ad3ba-09246238e4d11b-37c143e-100200-16cbc6055ae65; CNZZDATA1277864201=930369826-1566523222-%7C1566523222; .Cnblogs.AspNetCore.Cookies=CfDJ8BQYbW6Qx5RFuF4UTI7QvU1ZfPVntwKM1NzTaXOKfJrw-qVTaOroHYoX4UZpVjwwb1Dor-YLDPQ_jIMUz6v4tlqqdOt6j5mPrsfBb3ALvgoUiIFLIyPyjGzemiBOh7q3NRuf9Ppw2fpjnS7D6yu36dyEj_RHpadrshY7ieiPOSuSFmlmYZWUd_QcBLFtKthsCVq71DFt-YpAqEvodBvpE6cxIpi2pbAoKotrbWj5eopGesPjDQP9Pu3JbsBaH-Qow-y_a0wHjAsygVakYs5eMqiJ--r89ZCak2B0k_i5ZPPfWTdfNawGonTaEfiSeSLwIvDJvU64OIc3OgEqMkrR1l6xPC3THTS-kYQUSvx0Son4jOkNT-9BQuUCYiF7NsNR9fW5-ERZ1f9sOPFe4SldT4B3AS7aURcnb4NKaYP8J66ZHqzeiejcqh9PFoHJVozweXY6a2IHPZjvCcoWY9s9Fc4; .CNBlogsCookie=8F9E89719E435D522BA516B6BECEED8E19C600C5BAB0DBC7B4702273BFC4A227B2DD6A1EBD6A39A1CC301B7ABDF33CC9EF3FDA7AF06774D971D6527752FC031EFDF0A786E948650EACEE86339A3EF8503AF948B6; _gat=1"
}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'blog.middlewares.BlogSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   'blog.middlewares.BlogDownloaderMiddleware': 543,
}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   # 'blog.pipelines.BlogPipeline': 300,
	# 保存到redis中
	# 'blog.pipelines.BlogRedisPipeline': 300,

	# 保存到rabbitMQ
	'blog.pipelines.BlogMQPipeline': 300,

	# 分布式爬虫管道
    	# 'scrapy_redis.pipelines.RedisPipeline': 400
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

# 爬取文章的个数
CLOSESPIDER_PAGECOUNT = 1000

"""
# 分布式爬虫

# 增加了一个去重容器类的配置, 作用使用Redis的set集合来存储请求的指纹数据, 从而实现请求去重的持久化
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 使用scrapy-redis组件自己的调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 配置调度器是否要持久化, 也就是当爬虫结束了, 要不要清空Redis中请求队列和去重指纹的set。如果是True, 就表示要持久化存储, 就不清空数据, 否则清空数据
SCHEDULER_PERSIST = True


REDIS_HOST = '101.132.242.86'
REDIS_PORT = 6379
REDIS_ENCODING = 'utf-8'
"""

